package cn.lecosa.spark.kryo

import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import scala.collection.mutable.ArrayBuffer
import scala.util.Random
import org.apache.spark.storage.StorageLevel
import org.apache.spark.sql.SQLContext

case  class UserInfo(id:Long,name:String,gender:String,thirdParty:String) extends java.io.Serializable

object KryoTest {
  def main(args: Array[String]) {
    val start = System.currentTimeMillis()
    val conf = new SparkConf().setMaster("local[*]").setAppName("kryoTest")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .set("spark.kryo.registrator", "cn.lecosa.spark.kryo.MyRegisterKryo")
    val sc = new SparkContext(conf)
    val userInfo = sc.textFile("hdfs://park01:9000/home/kryo.txt").map(line=>
     line.split("\t")).map(user=> UserInfo(user(0).toLong, user(1), user(2), user(3)))
     
     val sqlContext=new SQLContext(sc)
      import sqlContext.implicits._
     userInfo.toDF().show()
    userInfo.persist(StorageLevel.MEMORY_AND_DISK_SER)
    println(userInfo.count())
    println(System.currentTimeMillis() - start)
    sc.stop()
  }
}